package com.access;

import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Date;

import org.htmlparser.Node;
import org.htmlparser.Parser;
import org.htmlparser.beans.LinkBean;
import org.htmlparser.beans.StringBean;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;

public class Spider {

	// extract child links excluding itself
	public ArrayList<String> extractChildLinks(String url) {

		LinkBean linkBean = new LinkBean();
		linkBean.setURL(url);
		URL[] childLinks = linkBean.getLinks();
		ArrayList<String> children = new ArrayList<String>();
		for (int i = 1; i < childLinks.length; i++) {
			children.add(childLinks[i].toString());
		}
		return children;
	}

	public String extractContent(String url) {

		StringBean stringBean;
		stringBean = new StringBean();
		stringBean.setLinks(false);
		stringBean.setURL(url);
		return stringBean.getStrings();

	}

	public String extractTitle(String url) throws ParserException, IOException {

		Parser parser = new Parser();
		parser.setURL(url);
		NodeList nodeList = parser.parse(new TagNameFilter("title"));
		Node node = nodeList.elementAt(0);
		if (node != null) {
			return node.toString().substring(7);
		} else {
			return null;
		}
		
	}

	public Date extractLastModifiedDate(String url)
			throws MalformedURLException, IOException {

		URLConnection connection = new URL(url).openConnection();
		connection.connect();
		long lastModified = connection.getLastModified();
		if (lastModified == 0) {
			return null;
		} else {
			return new Date(lastModified);
		}
	}

}
